{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Successfully downloaded train-images-idx3-ubyte.gz 9912422 bytes.\n",
      "Extracting MNIST/train-images-idx3-ubyte.gz\n",
      "Successfully downloaded train-labels-idx1-ubyte.gz 28881 bytes.\n",
      "Extracting MNIST/train-labels-idx1-ubyte.gz\n",
      "Successfully downloaded t10k-images-idx3-ubyte.gz 1648877 bytes.\n",
      "Extracting MNIST/t10k-images-idx3-ubyte.gz\n",
      "Successfully downloaded t10k-labels-idx1-ubyte.gz 4542 bytes.\n",
      "Extracting MNIST/t10k-labels-idx1-ubyte.gz\n",
      "Initializing parameters\n",
      "Step 50 | Loss: 208.06788635253906\n",
      "Step 100 | Loss: 180.06475830078125\n",
      "Step 150 | Loss: 203.51260375976562\n",
      "Step 200 | Loss: 145.33279418945312\n",
      "Step 250 | Loss: 152.0058135986328\n",
      "Step 300 | Loss: 132.89476013183594\n",
      "Step 350 | Loss: 143.39283752441406\n",
      "Step 400 | Loss: 140.32923889160156\n",
      "Step 450 | Loss: 134.5644989013672\n",
      "Step 500 | Loss: 137.3271026611328\n",
      "Step 550 | Loss: 139.12562561035156\n",
      "Step 600 | Loss: 130.7576904296875\n",
      "Step 650 | Loss: 128.86993408203125\n",
      "Step 700 | Loss: 121.57710266113281\n",
      "Step 750 | Loss: 121.71798706054688\n",
      "Step 800 | Loss: 132.4248809814453\n",
      "Step 850 | Loss: 134.16351318359375\n",
      "Step 900 | Loss: 128.24562072753906\n",
      "Step 950 | Loss: 122.6649398803711\n",
      "Step 1000 | Loss: 121.69792938232422\n",
      "Step 1050 | Loss: 125.6659927368164\n",
      "Step 1100 | Loss: 118.87915802001953\n",
      "Step 1150 | Loss: 125.58793640136719\n",
      "Step 1200 | Loss: 118.60771179199219\n",
      "Step 1250 | Loss: 123.4453353881836\n",
      "Step 1300 | Loss: 132.7337646484375\n",
      "Step 1350 | Loss: 123.17501831054688\n",
      "Step 1400 | Loss: 124.4575424194336\n",
      "Step 1450 | Loss: 118.01510620117188\n",
      "Step 1500 | Loss: 116.33611297607422\n",
      "Step 1550 | Loss: 129.4385528564453\n",
      "Step 1600 | Loss: 120.55036926269531\n",
      "Step 1650 | Loss: 130.09632873535156\n",
      "Step 1700 | Loss: 125.3007583618164\n",
      "Step 1750 | Loss: 122.07498931884766\n",
      "Step 1800 | Loss: 121.6435546875\n",
      "Step 1850 | Loss: 118.83889770507812\n",
      "Step 1900 | Loss: 119.53717041015625\n",
      "Step 1950 | Loss: 122.92294311523438\n",
      "Step 2000 | Loss: 120.94338989257812\n",
      "Step 2050 | Loss: 121.55541229248047\n",
      "Step 2100 | Loss: 118.6678237915039\n",
      "Step 2150 | Loss: 123.08565521240234\n",
      "Step 2200 | Loss: 125.24755859375\n",
      "Step 2250 | Loss: 115.99163818359375\n",
      "Step 2300 | Loss: 121.4999771118164\n",
      "Step 2350 | Loss: 119.26903533935547\n",
      "Step 2400 | Loss: 120.16402435302734\n",
      "Step 2450 | Loss: 113.10002136230469\n",
      "Step 2500 | Loss: 122.14546966552734\n",
      "Step 2550 | Loss: 122.36914825439453\n",
      "Step 2600 | Loss: 119.39669799804688\n",
      "Step 2650 | Loss: 124.0689468383789\n",
      "Step 2700 | Loss: 122.1514663696289\n",
      "Step 2750 | Loss: 120.1622543334961\n",
      "Step 2800 | Loss: 112.24446105957031\n",
      "Step 2850 | Loss: 107.25701904296875\n",
      "Step 2900 | Loss: 118.7703628540039\n",
      "Step 2950 | Loss: 125.22842407226562\n",
      "Step 3000 | Loss: 125.98233032226562\n",
      "Step 3050 | Loss: 122.34664154052734\n",
      "Step 3100 | Loss: 117.41297912597656\n",
      "Step 3150 | Loss: 120.1456069946289\n",
      "Step 3200 | Loss: 122.1562728881836\n",
      "Step 3250 | Loss: 115.88784790039062\n",
      "Step 3300 | Loss: 123.56607055664062\n",
      "Step 3350 | Loss: 120.23521423339844\n",
      "Step 3400 | Loss: 122.25651550292969\n",
      "Step 3450 | Loss: 114.21204376220703\n",
      "Step 3500 | Loss: 119.462890625\n",
      "Step 3550 | Loss: 120.56626892089844\n",
      "Step 3600 | Loss: 117.36223602294922\n",
      "Step 3650 | Loss: 110.40019226074219\n",
      "Step 3700 | Loss: 114.15618133544922\n",
      "Step 3750 | Loss: 116.40580749511719\n",
      "Step 3800 | Loss: 115.3848648071289\n",
      "Step 3850 | Loss: 119.44683837890625\n",
      "Step 3900 | Loss: 116.95742797851562\n",
      "Step 3950 | Loss: 118.3202133178711\n",
      "Step 4000 | Loss: 117.63365173339844\n",
      "Step 4050 | Loss: 112.71672058105469\n",
      "Step 4100 | Loss: 115.12691497802734\n",
      "Step 4150 | Loss: 116.4258804321289\n",
      "Step 4200 | Loss: 118.01799011230469\n",
      "Step 4250 | Loss: 115.61293029785156\n",
      "Step 4300 | Loss: 117.46105194091797\n",
      "Step 4350 | Loss: 118.05750274658203\n",
      "Step 4400 | Loss: 114.39136505126953\n",
      "Step 4450 | Loss: 116.23037719726562\n",
      "Step 4500 | Loss: 117.78424072265625\n",
      "Step 4550 | Loss: 118.01495361328125\n",
      "Step 4600 | Loss: 122.74853515625\n",
      "Step 4650 | Loss: 117.59159851074219\n",
      "Step 4700 | Loss: 113.462890625\n",
      "Step 4750 | Loss: 123.14982604980469\n",
      "Step 4800 | Loss: 116.58440399169922\n",
      "Step 4850 | Loss: 118.15299987792969\n",
      "Step 4900 | Loss: 122.93072509765625\n",
      "Step 4950 | Loss: 115.26317596435547\n",
      "Step 5000 | Loss: 119.47685241699219\n",
      "Step 5050 | Loss: 119.48384857177734\n",
      "Step 5100 | Loss: 118.50972747802734\n",
      "Step 5150 | Loss: 110.04559326171875\n",
      "Step 5200 | Loss: 116.88732147216797\n",
      "Step 5250 | Loss: 121.12173461914062\n",
      "Step 5300 | Loss: 122.22545623779297\n",
      "Step 5350 | Loss: 111.15548706054688\n",
      "Step 5400 | Loss: 110.47991943359375\n",
      "Step 5450 | Loss: 117.27287292480469\n",
      "Step 5500 | Loss: 120.08091735839844\n",
      "Step 5550 | Loss: 120.08263397216797\n",
      "Step 5600 | Loss: 112.765625\n",
      "Step 5650 | Loss: 117.5071792602539\n",
      "Step 5700 | Loss: 116.00777435302734\n",
      "Step 5750 | Loss: 117.9403305053711\n",
      "Step 5800 | Loss: 118.1498794555664\n",
      "Step 5850 | Loss: 115.70416259765625\n",
      "Step 5900 | Loss: 121.04488372802734\n",
      "Step 5950 | Loss: 117.7016372680664\n",
      "Step 6000 | Loss: 116.72579956054688\n",
      "Step 6050 | Loss: 117.94696044921875\n",
      "Step 6100 | Loss: 111.77839660644531\n",
      "Step 6150 | Loss: 117.66061401367188\n",
      "Step 6200 | Loss: 118.19465637207031\n",
      "Step 6250 | Loss: 119.76089477539062\n",
      "Step 6300 | Loss: 111.56034851074219\n",
      "Step 6350 | Loss: 115.82870483398438\n",
      "Step 6400 | Loss: 113.86763763427734\n",
      "Step 6450 | Loss: 116.8159408569336\n",
      "Step 6500 | Loss: 117.59591674804688\n",
      "Step 6550 | Loss: 119.50300598144531\n",
      "Step 6600 | Loss: 114.06906127929688\n",
      "Step 6650 | Loss: 117.50646209716797\n",
      "Step 6700 | Loss: 118.0813217163086\n",
      "Step 6750 | Loss: 114.482421875\n",
      "Step 6800 | Loss: 117.34271240234375\n",
      "Step 6850 | Loss: 114.98595428466797\n",
      "Step 6900 | Loss: 118.73905944824219\n",
      "Step 6950 | Loss: 113.58523559570312\n",
      "Step 7000 | Loss: 113.98788452148438\n",
      "Step 7050 | Loss: 121.17752075195312\n",
      "Step 7100 | Loss: 117.97523498535156\n",
      "Step 7150 | Loss: 121.0462875366211\n",
      "Step 7200 | Loss: 114.75642395019531\n",
      "Step 7250 | Loss: 116.09710693359375\n",
      "Step 7300 | Loss: 117.36449432373047\n",
      "Step 7350 | Loss: 117.70667266845703\n",
      "Step 7400 | Loss: 118.22289276123047\n",
      "Step 7450 | Loss: 113.93761444091797\n",
      "Step 7500 | Loss: 111.05577087402344\n",
      "Step 7550 | Loss: 123.77330017089844\n",
      "Step 7600 | Loss: 117.89901733398438\n",
      "Step 7650 | Loss: 117.20639038085938\n",
      "Step 7700 | Loss: 115.84173583984375\n",
      "Step 7750 | Loss: 117.5258560180664\n",
      "Step 7800 | Loss: 109.52965545654297\n",
      "Step 7850 | Loss: 118.94139862060547\n",
      "Step 7900 | Loss: 119.11836242675781\n",
      "Step 7950 | Loss: 113.66670227050781\n",
      "Step 8000 | Loss: 115.4374771118164\n",
      "Step 8050 | Loss: 110.00509643554688\n",
      "Step 8100 | Loss: 118.42096710205078\n",
      "Step 8150 | Loss: 117.70271301269531\n",
      "Step 8200 | Loss: 110.50149536132812\n",
      "Step 8250 | Loss: 116.86150360107422\n",
      "Step 8300 | Loss: 118.66195678710938\n",
      "Step 8350 | Loss: 115.85578155517578\n",
      "Step 8400 | Loss: 119.81375885009766\n",
      "Step 8450 | Loss: 117.7624282836914\n",
      "Step 8500 | Loss: 115.34953308105469\n",
      "Step 8550 | Loss: 117.8129653930664\n",
      "Step 8600 | Loss: 116.90750122070312\n",
      "Step 8650 | Loss: 111.59365844726562\n",
      "Step 8700 | Loss: 118.17662048339844\n",
      "Step 8750 | Loss: 120.67341613769531\n",
      "Step 8800 | Loss: 112.72423553466797\n",
      "Step 8850 | Loss: 119.2901382446289\n",
      "Step 8900 | Loss: 107.13825988769531\n",
      "Step 8950 | Loss: 118.46914672851562\n",
      "Step 9000 | Loss: 117.04972839355469\n",
      "Step 9050 | Loss: 116.61032104492188\n",
      "Step 9100 | Loss: 121.25959777832031\n",
      "Step 9150 | Loss: 117.86730194091797\n",
      "Step 9200 | Loss: 115.31476593017578\n",
      "Step 9250 | Loss: 115.94746398925781\n",
      "Step 9300 | Loss: 123.08782196044922\n",
      "Step 9350 | Loss: 118.65192413330078\n",
      "Step 9400 | Loss: 117.62959289550781\n",
      "Step 9450 | Loss: 113.04339599609375\n",
      "Step 9500 | Loss: 118.16273498535156\n",
      "Step 9550 | Loss: 119.29137420654297\n",
      "Step 9600 | Loss: 114.10884857177734\n",
      "Step 9650 | Loss: 116.94169616699219\n",
      "Step 9700 | Loss: 111.13204956054688\n",
      "Step 9750 | Loss: 116.00695037841797\n",
      "Step 9800 | Loss: 116.18476867675781\n",
      "Step 9850 | Loss: 112.22908020019531\n",
      "Step 9900 | Loss: 114.25606536865234\n",
      "Step 9950 | Loss: 108.73168182373047\n",
      "Step 10000 | Loss: 114.33287048339844\n",
      "Step 10050 | Loss: 121.33039093017578\n",
      "Step 10100 | Loss: 123.09110260009766\n",
      "Step 10150 | Loss: 119.38517761230469\n",
      "Step 10200 | Loss: 116.14971923828125\n",
      "Step 10250 | Loss: 121.53748321533203\n",
      "Step 10300 | Loss: 120.43973541259766\n",
      "Step 10350 | Loss: 116.59193420410156\n",
      "Step 10400 | Loss: 119.08804321289062\n",
      "Step 10450 | Loss: 113.619140625\n",
      "Step 10500 | Loss: 112.19953918457031\n",
      "Step 10550 | Loss: 111.6685562133789\n",
      "Step 10600 | Loss: 114.97622680664062\n",
      "Step 10650 | Loss: 119.60580444335938\n",
      "Step 10700 | Loss: 119.49198913574219\n",
      "Step 10750 | Loss: 118.43195343017578\n",
      "Step 10800 | Loss: 122.50007629394531\n",
      "Step 10850 | Loss: 116.86662292480469\n",
      "Step 10900 | Loss: 115.89854431152344\n",
      "Step 10950 | Loss: 118.72781372070312\n",
      "Step 11000 | Loss: 108.7948989868164\n",
      "Step 11050 | Loss: 118.50257873535156\n",
      "Step 11100 | Loss: 110.60926055908203\n",
      "Step 11150 | Loss: 120.69537353515625\n",
      "Step 11200 | Loss: 114.28993225097656\n",
      "Step 11250 | Loss: 115.44013977050781\n",
      "Step 11300 | Loss: 110.51482391357422\n",
      "Step 11350 | Loss: 117.07335662841797\n",
      "Step 11400 | Loss: 117.82571411132812\n",
      "Step 11450 | Loss: 116.74909210205078\n",
      "Step 11500 | Loss: 116.3514633178711\n",
      "Step 11550 | Loss: 118.20392608642578\n",
      "Step 11600 | Loss: 118.54083251953125\n",
      "Step 11650 | Loss: 120.45144653320312\n",
      "Step 11700 | Loss: 112.87490844726562\n",
      "Step 11750 | Loss: 114.39193725585938\n",
      "Step 11800 | Loss: 114.47300720214844\n",
      "Step 11850 | Loss: 114.05213165283203\n",
      "Step 11900 | Loss: 115.93722534179688\n",
      "Step 11950 | Loss: 115.79370880126953\n",
      "Step 12000 | Loss: 121.48307800292969\n",
      "Step 12050 | Loss: 112.57547760009766\n",
      "Step 12100 | Loss: 114.92308807373047\n",
      "Step 12150 | Loss: 118.25947570800781\n",
      "Step 12200 | Loss: 114.76628875732422\n",
      "Step 12250 | Loss: 113.80509948730469\n",
      "Step 12300 | Loss: 124.15151977539062\n",
      "Step 12350 | Loss: 122.06372833251953\n",
      "Step 12400 | Loss: 117.94454193115234\n",
      "Step 12450 | Loss: 117.29557800292969\n",
      "Step 12500 | Loss: 112.87556457519531\n",
      "Step 12550 | Loss: 119.79658508300781\n",
      "Step 12600 | Loss: 117.26377868652344\n",
      "Step 12650 | Loss: 116.1286392211914\n",
      "Step 12700 | Loss: 119.35836029052734\n",
      "Step 12750 | Loss: 118.67609405517578\n",
      "Step 12800 | Loss: 118.87919616699219\n",
      "Step 12850 | Loss: 119.9450912475586\n",
      "Step 12900 | Loss: 117.23912048339844\n",
      "Step 12950 | Loss: 116.15021514892578\n",
      "Step 13000 | Loss: 116.6678695678711\n",
      "Step 13050 | Loss: 119.83767700195312\n",
      "Step 13100 | Loss: 110.35836029052734\n",
      "Step 13150 | Loss: 108.51178741455078\n",
      "Step 13200 | Loss: 115.44359588623047\n",
      "Step 13250 | Loss: 115.70832061767578\n",
      "Step 13300 | Loss: 117.70867156982422\n",
      "Step 13350 | Loss: 111.4530258178711\n",
      "Step 13400 | Loss: 125.64757537841797\n",
      "Step 13450 | Loss: 111.11370849609375\n",
      "Step 13500 | Loss: 119.33474731445312\n",
      "Step 13550 | Loss: 115.30855560302734\n",
      "Step 13600 | Loss: 110.55035400390625\n",
      "Step 13650 | Loss: 116.16586303710938\n",
      "Step 13700 | Loss: 121.72457122802734\n",
      "Step 13750 | Loss: 117.2103500366211\n",
      "Step 13800 | Loss: 114.72425079345703\n",
      "Step 13850 | Loss: 110.55821228027344\n",
      "Step 13900 | Loss: 111.8604507446289\n",
      "Step 13950 | Loss: 116.0633316040039\n",
      "Step 14000 | Loss: 114.9267349243164\n",
      "Step 14050 | Loss: 114.90238189697266\n",
      "Step 14100 | Loss: 113.99726867675781\n",
      "Step 14150 | Loss: 118.53600311279297\n",
      "Step 14200 | Loss: 118.81285095214844\n",
      "Step 14250 | Loss: 124.75857543945312\n",
      "Step 14300 | Loss: 119.23839569091797\n",
      "Step 14350 | Loss: 114.2392578125\n",
      "Step 14400 | Loss: 119.28097534179688\n",
      "Step 14450 | Loss: 112.29423522949219\n",
      "Step 14500 | Loss: 117.7653579711914\n",
      "Step 14550 | Loss: 112.31188201904297\n",
      "Step 14600 | Loss: 120.16435241699219\n",
      "Step 14650 | Loss: 115.26638793945312\n",
      "Step 14700 | Loss: 118.45664978027344\n",
      "Step 14750 | Loss: 119.6676254272461\n",
      "Step 14800 | Loss: 116.76998901367188\n",
      "Step 14850 | Loss: 114.6698989868164\n",
      "Step 14900 | Loss: 120.42974853515625\n",
      "Step 14950 | Loss: 121.3724594116211\n",
      "Step 15000 | Loss: 114.86787414550781\n",
      "Step 15050 | Loss: 120.95048522949219\n",
      "Step 15100 | Loss: 118.16832733154297\n",
      "Step 15150 | Loss: 110.7814712524414\n",
      "Step 15200 | Loss: 119.54023742675781\n",
      "Step 15250 | Loss: 119.33824920654297\n",
      "Step 15300 | Loss: 119.40328216552734\n",
      "Step 15350 | Loss: 118.04212951660156\n",
      "Step 15400 | Loss: 113.81212615966797\n",
      "Step 15450 | Loss: 110.48765563964844\n",
      "Step 15500 | Loss: 116.61421966552734\n",
      "Step 15550 | Loss: 110.74839782714844\n",
      "Step 15600 | Loss: 112.31326293945312\n",
      "Step 15650 | Loss: 115.25269317626953\n",
      "Step 15700 | Loss: 123.32250213623047\n",
      "Step 15750 | Loss: 114.82608032226562\n",
      "Step 15800 | Loss: 122.91587829589844\n",
      "Step 15850 | Loss: 121.05029296875\n",
      "Step 15900 | Loss: 117.34131622314453\n",
      "Step 15950 | Loss: 115.32041931152344\n",
      "Step 16000 | Loss: 111.40978240966797\n",
      "Step 16050 | Loss: 114.45780181884766\n",
      "Step 16100 | Loss: 114.42517852783203\n",
      "Step 16150 | Loss: 119.44152069091797\n",
      "Step 16200 | Loss: 118.29029083251953\n",
      "Step 16250 | Loss: 114.74148559570312\n",
      "Step 16300 | Loss: 117.47858428955078\n",
      "Step 16350 | Loss: 113.23002624511719\n",
      "Step 16400 | Loss: 109.5634994506836\n",
      "Step 16450 | Loss: 115.4550552368164\n",
      "Step 16500 | Loss: 118.49871063232422\n",
      "Step 16550 | Loss: 115.15196228027344\n",
      "Step 16600 | Loss: 116.4708480834961\n",
      "Step 16650 | Loss: 111.94804382324219\n",
      "Step 16700 | Loss: 113.8924789428711\n",
      "Step 16750 | Loss: 123.36951446533203\n",
      "Step 16800 | Loss: 114.1649398803711\n",
      "Step 16850 | Loss: 115.64714813232422\n",
      "Step 16900 | Loss: 117.13810729980469\n",
      "Step 16950 | Loss: 117.12565612792969\n",
      "Step 17000 | Loss: 119.39795684814453\n",
      "Step 17050 | Loss: 117.11557006835938\n",
      "Step 17100 | Loss: 119.05950164794922\n",
      "Step 17150 | Loss: 118.0962905883789\n",
      "Step 17200 | Loss: 115.04369354248047\n",
      "Step 17250 | Loss: 115.11548614501953\n",
      "Step 17300 | Loss: 119.72411346435547\n",
      "Step 17350 | Loss: 121.55140686035156\n",
      "Step 17400 | Loss: 112.74524688720703\n",
      "Step 17450 | Loss: 118.12261962890625\n",
      "Step 17500 | Loss: 112.08924102783203\n",
      "Step 17550 | Loss: 120.76656341552734\n",
      "Step 17600 | Loss: 112.10700988769531\n",
      "Step 17650 | Loss: 114.62565612792969\n",
      "Step 17700 | Loss: 120.84262084960938\n",
      "Step 17750 | Loss: 110.32308959960938\n",
      "Step 17800 | Loss: 115.84912109375\n",
      "Step 17850 | Loss: 112.08287048339844\n"
     ]
    }
   ],
   "source": [
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "import os.path\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "mnist = input_data.read_data_sets('MNIST')\n",
    "\n",
    "input_dim = 784\n",
    "hidden_encoder_dim = 400\n",
    "hidden_decoder_dim = 400\n",
    "latent_dim = 20\n",
    "lam = 0\n",
    "\n",
    "def weight_variable(shape):\n",
    "    initial = tf.truncated_normal(shape, stddev=0.001)\n",
    "    return tf.Variable(initial)\n",
    "\n",
    "def bias_variable(shape):\n",
    "    initial = tf.constant(0., shape=shape)\n",
    "    return tf.Variable(initial)\n",
    "\n",
    "x = tf.placeholder(\"float\", shape=[None, input_dim])\n",
    "l2_loss = tf.constant(0.0)\n",
    "\n",
    "W_encoder_input_hidden = weight_variable([input_dim,hidden_encoder_dim])\n",
    "b_encoder_input_hidden = bias_variable([hidden_encoder_dim])\n",
    "l2_loss += tf.nn.l2_loss(W_encoder_input_hidden)\n",
    "\n",
    "# Hidden layer encoder\n",
    "hidden_encoder = tf.nn.relu(tf.matmul(x, W_encoder_input_hidden) + b_encoder_input_hidden)\n",
    "\n",
    "W_encoder_hidden_mu = weight_variable([hidden_encoder_dim,latent_dim])\n",
    "b_encoder_hidden_mu = bias_variable([latent_dim])\n",
    "l2_loss += tf.nn.l2_loss(W_encoder_hidden_mu)\n",
    "\n",
    "# Mu encoder\n",
    "mu_encoder = tf.matmul(hidden_encoder, W_encoder_hidden_mu) + b_encoder_hidden_mu\n",
    "\n",
    "W_encoder_hidden_logvar = weight_variable([hidden_encoder_dim,latent_dim])\n",
    "b_encoder_hidden_logvar = bias_variable([latent_dim])\n",
    "l2_loss += tf.nn.l2_loss(W_encoder_hidden_logvar)\n",
    "\n",
    "# Sigma encoder\n",
    "logvar_encoder = tf.matmul(hidden_encoder, W_encoder_hidden_logvar) + b_encoder_hidden_logvar\n",
    "\n",
    "# Sample epsilon\n",
    "epsilon = tf.random_normal(tf.shape(logvar_encoder), name='epsilon')\n",
    "\n",
    "# Sample latent variable\n",
    "std_encoder = tf.exp(0.5 * logvar_encoder)\n",
    "z = mu_encoder + tf.multiply(std_encoder, epsilon)\n",
    "\n",
    "W_decoder_z_hidden = weight_variable([latent_dim,hidden_decoder_dim])\n",
    "b_decoder_z_hidden = bias_variable([hidden_decoder_dim])\n",
    "l2_loss += tf.nn.l2_loss(W_decoder_z_hidden)\n",
    "\n",
    "# Hidden layer decoder\n",
    "hidden_decoder = tf.nn.relu(tf.matmul(z, W_decoder_z_hidden) + b_decoder_z_hidden)\n",
    "\n",
    "W_decoder_hidden_reconstruction = weight_variable([hidden_decoder_dim, input_dim])\n",
    "b_decoder_hidden_reconstruction = bias_variable([input_dim])\n",
    "l2_loss += tf.nn.l2_loss(W_decoder_hidden_reconstruction)\n",
    "\n",
    "KLD = -0.5 * tf.reduce_sum(1 + logvar_encoder - tf.pow(mu_encoder, 2) - tf.exp(logvar_encoder), reduction_indices=1)\n",
    "\n",
    "x_hat = tf.matmul(hidden_decoder, W_decoder_hidden_reconstruction) + b_decoder_hidden_reconstruction\n",
    "BCE = tf.reduce_sum(tf.nn.sigmoid_cross_entropy_with_logits(logits=x_hat, labels=x), reduction_indices=1)\n",
    "\n",
    "loss = tf.reduce_mean(BCE + KLD)\n",
    "\n",
    "regularized_loss = loss + lam * l2_loss\n",
    "\n",
    "loss_summ = tf.summary.scalar(\"lowerbound\", loss)\n",
    "train_step = tf.train.AdamOptimizer(0.01).minimize(regularized_loss)\n",
    "\n",
    "# add op for merging summary\n",
    "summary_op = tf.summary.merge_all()\n",
    "\n",
    "# add Saver ops\n",
    "saver = tf.train.Saver()\n",
    "\n",
    "n_steps = int(1e6)\n",
    "batch_size = 100\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    summary_writer = tf.summary.FileWriter('experiment',\n",
    "                                          graph=sess.graph)\n",
    "  \n",
    "    print(\"Initializing parameters\")\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    \n",
    "    for step in range(1, n_steps):\n",
    "        batch = mnist.train.next_batch(batch_size)\n",
    "        feed_dict = {x: batch[0]}\n",
    "        _, cur_loss, summary_str = sess.run([train_step, loss, summary_op], feed_dict=feed_dict)\n",
    "        summary_writer.add_summary(summary_str, step)\n",
    "\n",
    "        if step % 50 == 0:\n",
    "            print(\"Step {0} | Loss: {1}\".format(step, cur_loss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import collections\n",
    "collections."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
